package com.dong.webMagic;

import com.dong.common.lang.StringUtils;
import com.dong.webMagic.pipeline.HupuNewsPipeline;
import com.dong.webMagic.pipeline.SoutPipeline;
import org.apache.ibatis.io.Resources;
import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.apache.ibatis.session.SqlSessionFactoryBuilder;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

import java.io.IOException;
import java.io.InputStream;

public class HupuNewsSpider implements PageProcessor {
    // 抓取网站的相关配置，包括编码、抓取间隔、重试次数等
    private Site site = Site.me().setRetryTimes(3).setSleepTime(100);

    @Override
    public Site getSite() {
        return site;
    }

    @Override
    public void process(Page page) {
        // 文章页，匹配 https://voice.hupu.com/nba/七位数字.html
        if (page.getUrl().regex("https://voice\\.hupu\\.com/nba/[0-9]{7}\\.html").match()) {
            String url = page.getUrl().get();
            page.putField("id", url.substring(url.lastIndexOf("/") + 1, url.length() - 5));
            page.putField("type", url.substring(StringUtils.getFromIndex(url, "/", 3) + 1, StringUtils.getFromIndex(url, "/", 4)));
            page.putField("title", page.getHtml().xpath("/html/body/div[4]/div[1]/div[1]/h1/text()").toString());
            page.putField("content", page.getHtml().xpath("/html/body/div[4]/div[1]/div[2]/div/div[2]/p/text()").all().toString());
            page.putField("createTime", page.getHtml().xpath("/html/body/div[4]/div[1]/div[1]/div[1]/span[1]/a/span/text()").toString());
        }
        // 列表页
        else {
            // 文章url
            page.addTargetRequests(page.getHtml().xpath("/html/body/div[3]/div[1]/div[2]/ul/li/div[1]/h4/a/@href").all());
            // 翻页url
            page.addTargetRequests(page.getHtml().xpath("/html/body/div[3]/div[1]/div[3]/a[@class='page-btn-prev']/@href").all());
        }
    }

    public static void main(String[] args) {
        String resource = "mybatis-config.xml";// 定义配置文件路径
        InputStream inputStream = null;
        try {
            inputStream = Resources.getResourceAsStream(resource);// 读取配置文件
        } catch (IOException e) {
            e.printStackTrace();
        }
        SqlSessionFactory sqlSessionFactory = new SqlSessionFactoryBuilder().build(inputStream);// 注册mybatis工厂
        SqlSession sqlSession = sqlSessionFactory.openSession();// 得到连接对象

        Spider.create(new HupuNewsSpider()).addUrl("https://voice.hupu.com/nba/1").addPipeline(new HupuNewsPipeline(sqlSession))
                .thread(3).run();

        sqlSession.close();
    }
}